{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/workspace/mdy/miniforge/envs/mdy/lib/python3.10/site-packages/_distutils_hack/__init__.py:53: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import triton\n",
    "import triton.language as tl\n",
    "from copy import deepcopy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 失败作品\n",
    "- 这是一个失败的作品，将矩阵乘和silu融为一个算子，但是效果很差，因此最好不要把矩阵乘的操作和元素级的操作相结合\n",
    "- 但是梯度之类都是对的， 仅供参考，DW还没实现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "@triton.jit\n",
    "def _fused_matmul_silu_fwd(\n",
    "        # Pointers to matrices\n",
    "        X, W, Y, UP, GATE, ACT,\n",
    "        # Matrix dimensions\n",
    "        M, N, K,\n",
    "        # The stride variables represent how much to increase the ptr by when moving by 1\n",
    "        # element in a particular dimension. E.g. `stride_am` is how much to increase `a_ptr`\n",
    "        # by to get the element one row down (A has M rows).\n",
    "        stride_am, stride_ak,  #\n",
    "        stride_bk, stride_bn,  #\n",
    "        stride_cm, stride_cn,\n",
    "        # Meta-parameters\n",
    "        BLOCK_SIZE_M: tl.constexpr, BLOCK_SIZE_N: tl.constexpr, BLOCK_SIZE_K: tl.constexpr,  #\n",
    "        DTYPE: tl.constexpr  #\n",
    "):\n",
    "    \"\"\"Kernel for computing the matmul C = A x B.\n",
    "    A has shape (M, K), B has shape (K, N) and C has shape (M, N)\n",
    "    \"\"\"\n",
    "    pid = tl.program_id(axis=0)\n",
    "    num_block_n = tl.cdiv(N, BLOCK_SIZE_N) // 2\n",
    "    pid_m = pid // num_block_n\n",
    "    pid_n = pid % num_block_n\n",
    "    # a_ptrs += pid_m * stride_am * BLOCK_SIZE_M\n",
    "    # n_offset = pid_n * stride_bn * BLOCK_SIZE_N\n",
    "\n",
    "    x_block_ptrs = tl.make_block_ptr(\n",
    "        base=X,\n",
    "        shape=(M, K),\n",
    "        strides=(stride_am, stride_ak),\n",
    "        offsets=(pid_m * BLOCK_SIZE_M, 0),\n",
    "        block_shape=(BLOCK_SIZE_M, BLOCK_SIZE_K),\n",
    "        order=(1,0)\n",
    "    )\n",
    "\n",
    "    w_up_block_ptrs = tl.make_block_ptr(\n",
    "        base=W,\n",
    "        shape=(K, N),\n",
    "        strides=(stride_bk, stride_bn),\n",
    "        offsets=(0, pid_n*BLOCK_SIZE_N),\n",
    "        block_shape=(BLOCK_SIZE_K, BLOCK_SIZE_N),\n",
    "        order=(1,0)\n",
    "    )\n",
    "\n",
    "    w_gate_block_ptrs = tl.make_block_ptr(\n",
    "        base=W,\n",
    "        shape=(K, N),\n",
    "        strides=(stride_bk, stride_bn),\n",
    "        offsets=(0, (pid_n + num_block_n)*BLOCK_SIZE_N),\n",
    "        block_shape=(BLOCK_SIZE_K, BLOCK_SIZE_N),\n",
    "        order=(1,0)\n",
    "    )\n",
    "\n",
    "    y_block_ptrs = tl.make_block_ptr(\n",
    "        base=Y,\n",
    "        shape=(M, N//2),\n",
    "        strides=(stride_cm, stride_cn),\n",
    "        offsets=(pid_m*BLOCK_SIZE_M, pid_n*BLOCK_SIZE_N),\n",
    "        block_shape=(BLOCK_SIZE_M, BLOCK_SIZE_N),\n",
    "        order=(1,0)\n",
    "    )\n",
    "\n",
    "    gate_block_ptrs = tl.make_block_ptr(\n",
    "        base=GATE,\n",
    "        shape=(M, N//2),\n",
    "        strides=(stride_cm, stride_cn),\n",
    "        offsets=(pid_m*BLOCK_SIZE_M, pid_n*BLOCK_SIZE_N),\n",
    "        block_shape=(BLOCK_SIZE_M, BLOCK_SIZE_N),\n",
    "        order=(1,0)\n",
    "    )\n",
    "\n",
    "    up_block_ptrs = tl.make_block_ptr(\n",
    "        base=UP,\n",
    "        shape=(M, N//2),\n",
    "        strides=(stride_cm, stride_cn),\n",
    "        offsets=(pid_m*BLOCK_SIZE_M, pid_n*BLOCK_SIZE_N),\n",
    "        block_shape=(BLOCK_SIZE_M, BLOCK_SIZE_N),\n",
    "        order=(1,0)\n",
    "    )\n",
    "\n",
    "    act_block_ptrs = tl.make_block_ptr(\n",
    "        base=ACT,\n",
    "        shape=(M, N//2),\n",
    "        strides=(stride_cm, stride_cn),\n",
    "        offsets=(pid_m*BLOCK_SIZE_M, pid_n*BLOCK_SIZE_N),\n",
    "        block_shape=(BLOCK_SIZE_M, BLOCK_SIZE_N),\n",
    "        order=(1,0)\n",
    "    )\n",
    "    \n",
    "    dtype = tl.float32\n",
    "    if DTYPE == 'bf16':\n",
    "        dtype = tl.bfloat16\n",
    "    elif DTYPE == 'fp16':\n",
    "        dtype = tl.float16\n",
    "\n",
    "    acc_up = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)\n",
    "    acc_gate = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)\n",
    "    for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)):\n",
    "        # Load the next block of A and B, generate a mask by checking the K dimension.\n",
    "        # If it is out of bounds, set it to 0.\n",
    "        x = tl.load(x_block_ptrs, boundary_check=(0,), padding_option='zero')\n",
    "        w_up = tl.load(w_up_block_ptrs) # 理论上b不会溢出，相当于权重w，dim都是2的指数倍或者BLOCK_SIZE的整数倍\n",
    "        w_gate = tl.load(w_gate_block_ptrs)\n",
    "        # We accumulate along the K dimension.\n",
    "        acc_up = tl.dot(x, w_up, acc_up)\n",
    "        acc_gate = tl.dot(x, w_gate, acc_gate)\n",
    "        # Advance the ptrs to the next K block.\n",
    "        x_block_ptrs = tl.advance(x_block_ptrs, offsets=(0, BLOCK_SIZE_K))\n",
    "        w_up_block_ptrs = tl.advance(w_up_block_ptrs, offsets=(BLOCK_SIZE_K, 0))\n",
    "        w_gate_block_ptrs = tl.advance(w_gate_block_ptrs, offsets=(BLOCK_SIZE_K, 0))\n",
    "\n",
    "    tl.store(up_block_ptrs, acc_up.to(dtype), boundary_check=(0,))\n",
    "    tl.store(gate_block_ptrs, acc_gate.to(dtype), boundary_check=(0,))\n",
    "    # You can fuse arbitrary activation functions here\n",
    "    # while the accumulator is still in FP32!\n",
    "    act =  acc_gate * tl.sigmoid(acc_gate)\n",
    "    y = act * acc_up\n",
    "    # tl.store(up_block_ptrs, acc_up.to(dtype), boundary_check=(0,))\n",
    "    # tl.store(gate_block_ptrs, acc_gate.to(dtype), boundary_check=(0,))\n",
    "    tl.store(act_block_ptrs, act.to(dtype), boundary_check=(0,))\n",
    "    tl.store(y_block_ptrs, y.to(dtype), boundary_check=(0,)) # M是bs * seq_len， 可能会溢出\n",
    "\n",
    "\n",
    "@triton.jit\n",
    "def _fused_mul_silu_bwd_dupgateact(UP, GATE, ACT, \n",
    "                               DY, DUP, DGATE,\n",
    "                               stride_m, stride_n,\n",
    "                               N, BLOCK_N: tl.constexpr\n",
    "                               ):\n",
    "    pid = tl.program_id(0)\n",
    "    offset = pid * stride_m\n",
    "    cols = tl.arange(0, BLOCK_N)\n",
    "    ptrs = offset + cols\n",
    "    mask = cols < N\n",
    "    \n",
    "    dy = tl.load(DY+ptrs, mask=mask, other=0.)\n",
    "    dtype = dy.dtype\n",
    "    dy = dy.to(tl.float32)\n",
    "    act = tl.load(ACT+ptrs, mask=mask, other=0.).to(tl.float32)\n",
    "    dup = act * dy\n",
    "    tl.store(DUP+ptrs, dup.to(dtype), mask=mask)\n",
    "\n",
    "    gate = tl.load(GATE+ptrs, mask=mask, other=0.).to(tl.float32)\n",
    "    up = tl.load(UP+ptrs, mask=mask, other=0.).to(tl.float32)\n",
    "    dact = up * dy\n",
    "    gate_neg_exp = tl.exp(-gate)\n",
    "    tmp = 1 + gate_neg_exp\n",
    "    fenzi =  tmp + gate * gate_neg_exp\n",
    "    fenmu = tmp * tmp\n",
    "    dgate = (fenzi / fenmu) * dact\n",
    "    tl.store(DGATE+ptrs, dgate.to(dtype), mask=mask)\n",
    "\n",
    "@triton.jit\n",
    "def _fused_mul_silu_bwd_dx(W,\n",
    "                    DUP, DGATE, DX, \n",
    "                    M, K, N, \n",
    "                    stride_upm, stride_upn, \n",
    "                    stride_wk, stride_wn, \n",
    "                    stride_xm, stride_xk,\n",
    "                    BLOCK_SIZE_M: tl.constexpr, BLOCK_SIZE_K: tl.constexpr, BLOCK_SIZE_N: tl.constexpr, \n",
    "                    DTYPE: tl.constexpr\n",
    "                    ):\n",
    "    pid = tl.program_id(0)\n",
    "    num_block_k = tl.cdiv(K, BLOCK_SIZE_K)\n",
    "    pid_m = pid // num_block_k\n",
    "    pid_k = pid % num_block_k\n",
    "\n",
    "    dup_block_ptrs = tl.make_block_ptr(\n",
    "        base=DUP,\n",
    "        shape=(M, N),\n",
    "        strides=(stride_upm, stride_upn),\n",
    "        offsets=(pid_m * BLOCK_SIZE_M, 0),\n",
    "        block_shape=(BLOCK_SIZE_M, BLOCK_SIZE_N),\n",
    "        order=(1,0)\n",
    "    )\n",
    "\n",
    "    dgate_block_ptrs = tl.make_block_ptr(\n",
    "        base=DGATE,\n",
    "        shape=(M, N),\n",
    "        strides=(stride_upm, stride_upn),\n",
    "        offsets=(pid_m * BLOCK_SIZE_M, 0),\n",
    "        block_shape=(BLOCK_SIZE_M, BLOCK_SIZE_N),\n",
    "        order=(1,0)\n",
    "    )\n",
    "\n",
    "    w_up_block_ptrs = tl.make_block_ptr(\n",
    "        base=W,\n",
    "        shape=(2*N, K),\n",
    "        strides=(stride_wn, stride_wk),\n",
    "        offsets=(0, pid_k*BLOCK_SIZE_K),\n",
    "        block_shape=(BLOCK_SIZE_N, BLOCK_SIZE_K),\n",
    "        order=(0,1)\n",
    "    )\n",
    "\n",
    "    w_gate_block_ptrs = tl.make_block_ptr(\n",
    "        base=W,\n",
    "        shape=(2*N, K),\n",
    "        strides=(stride_wn, stride_wk),\n",
    "        offsets=(N, pid_k*BLOCK_SIZE_K),\n",
    "        block_shape=(BLOCK_SIZE_N, BLOCK_SIZE_K),\n",
    "        order=(0,1)\n",
    "    )\n",
    "\n",
    "    dx_block_ptrs = tl.make_block_ptr(\n",
    "        base=DX,\n",
    "        shape=(M, K),\n",
    "        strides=(stride_xm, stride_xk),\n",
    "        offsets=(pid_m*BLOCK_SIZE_M, pid_k*BLOCK_SIZE_K),\n",
    "        block_shape=(BLOCK_SIZE_M, BLOCK_SIZE_K),\n",
    "        order=(1,0)\n",
    "    )\n",
    "\n",
    "    dtype = tl.float32\n",
    "    if DTYPE == 'bf16':\n",
    "        dtype = tl.bfloat16\n",
    "    elif DTYPE == 'fp16':\n",
    "        dtype = tl.float16\n",
    "\n",
    "    dx = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_K), dtype=tl.float32)\n",
    "    for _ in range(0, N, BLOCK_SIZE_N):\n",
    "        dup = tl.load(dup_block_ptrs, boundary_check=(1,), padding_option='zero')\n",
    "        dgate = tl.load(dgate_block_ptrs, boundary_check=(1,), padding_option='zero')\n",
    "        w_up = tl.load(w_up_block_ptrs)\n",
    "        w_gate = tl.load(w_gate_block_ptrs)\n",
    "        dx += tl.dot(dup, w_up)\n",
    "        dx += tl.dot(dgate, w_gate)\n",
    "        dup_block_ptrs = tl.advance(dup_block_ptrs, (0, BLOCK_SIZE_N))\n",
    "        dgate_block_ptrs = tl.advance(dgate_block_ptrs, (0, BLOCK_SIZE_N))\n",
    "        w_up_block_ptrs = tl.advance(w_up_block_ptrs, (BLOCK_SIZE_N, 0))\n",
    "        w_gate_block_ptrs = tl.advance(w_gate_block_ptrs, (BLOCK_SIZE_N, 0))\n",
    "\n",
    "    tl.store(dx_block_ptrs, dx.to(dtype), boundary_check=(0,))\n",
    "\n",
    "tmp = []\n",
    "class _FusedMulSiLU(torch.autograd.Function):\n",
    "    @staticmethod\n",
    "    def forward(ctx, x, weight):\n",
    "        dtype = x.dtype\n",
    "        DTYPE = 'fp32'\n",
    "        if dtype == torch.float16:\n",
    "            DTYPE = 'fp16'\n",
    "        elif dtype == torch.bfloat16:\n",
    "            DTYPE = 'bf16'   \n",
    "        # Check constraints.\n",
    "        input_shape = x.shape\n",
    "        x = x.view(-1, input_shape[-1])\n",
    "        M, K = x.shape\n",
    "        K, N = weight.shape\n",
    "        # Allocates output.\n",
    "        y = torch.empty((M, N//2), device=x.device, dtype=dtype)\n",
    "        up = torch.empty((M, N//2), device=x.device, dtype=dtype)\n",
    "        gate = torch.empty((M, N//2), device=x.device, dtype=dtype)\n",
    "        act = torch.empty((M, N//2), device=x.device, dtype=dtype)\n",
    "        # 1D launch kernel where each block gets its own program.\n",
    "        BLOCK_SIZE_M = min(64, triton.next_power_of_2(M))\n",
    "        BLOCK_SIZE_N = min(64, triton.next_power_of_2(N))\n",
    "        BLOCK_SIZE_K = 16\n",
    "        num_warps = 4\n",
    "        num_stages = 4\n",
    "        grid = lambda META: (triton.cdiv(M, META['BLOCK_SIZE_M']) * triton.cdiv(N, META['BLOCK_SIZE_N']), )\n",
    "        _fused_matmul_silu_fwd[grid](\n",
    "            x, weight, y, up, gate, act,#\n",
    "            M, N, K,  #\n",
    "            x.stride(0), x.stride(1),  #\n",
    "            weight.stride(0), weight.stride(1),  #\n",
    "            y.stride(0), y.stride(1),  #\n",
    "            BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, DTYPE,\n",
    "            num_warps=num_warps, num_stages=num_stages, \n",
    "        )\n",
    "        ctx.save_for_backward(x, weight, up, gate, act)\n",
    "        ctx.input_shape = input_shape\n",
    "        ctx.DTYPE = DTYPE\n",
    "        # ctx.BLOCK_SIZE = (BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K)\n",
    "        return y.view(*input_shape[:-1],-1)\n",
    "        # return act\n",
    "    \n",
    "    @staticmethod\n",
    "    def backward(ctx, dy):\n",
    "        input_shape = ctx.input_shape\n",
    "        dy = dy.view(*input_shape[:-1], -1)\n",
    "        x, w, up, gate, act = ctx.saved_tensors # w.shape: [K, N]\n",
    "        dup = torch.empty_like(up)\n",
    "        dgate = torch.empty_like(gate)\n",
    "        # \n",
    "        #  ************    first_step: compute dup dgate    ************\n",
    "        M, N = dy.shape\n",
    "        BLOCK_N = triton.next_power_of_2(N)\n",
    "        _fused_mul_silu_bwd_dupgateact[(M,)](up, gate, act, \n",
    "                                   dy, dup, dgate,\n",
    "                                   *dy.stride(),\n",
    "                                   N, BLOCK_N, \n",
    "                                   num_warps=8, num_stages=4)\n",
    "        dx = torch.empty_like(x)\n",
    "        K = x.shape[-1]\n",
    "        # print(M,N,K)\n",
    "        BLOCK_SIZE_M = min(32, triton.next_power_of_2(M))\n",
    "        BLOCK_SIZE_K = min(32, triton.next_power_of_2(K))\n",
    "        BLOCK_SIZE_N = 32\n",
    "        num_warps = 4\n",
    "        num_stages = 4\n",
    "        # print(dup[:3])\n",
    "        # print(dgate[:3])\n",
    "        # print(w.shape)\n",
    "        # print(w.stride(), dup.stride(), dx.stride())\n",
    "        grid = lambda META: (triton.cdiv(M, META['BLOCK_SIZE_M']) * triton.cdiv(K, META['BLOCK_SIZE_K']), )\n",
    "        _fused_mul_silu_bwd_dx[grid](w,\n",
    "                    dup, dgate, dx,\n",
    "                    M, K, N, \n",
    "                    *dup.stride(),\n",
    "                    *w.stride(), \n",
    "                    *dx.stride(),\n",
    "                    BLOCK_SIZE_M, BLOCK_SIZE_K, BLOCK_SIZE_N, \n",
    "                    ctx.DTYPE, \n",
    "                    num_stages=num_stages, num_warps=num_warps,\n",
    "                    )\n",
    "        # dx = torch.cat([dup, dgate], axis=-1) @ w.T\n",
    "        return dx, None\n",
    "\n",
    "fused_mul_silu = _FusedMulSiLU.apply\n",
    "\n",
    "class MulSiLU(torch.nn.Module):\n",
    "    def __init__(self, hidden_size, intermediate_size=None):\n",
    "        super().__init__()\n",
    "        if intermediate_size is None:\n",
    "            intermediate_size = hidden_size * 8\n",
    "        self.w = torch.nn.Linear(hidden_size, intermediate_size, bias=False)\n",
    "        self.act_fn = torch.nn.SiLU()\n",
    "\n",
    "    def forward(self, hidden_state):\n",
    "        out = self.w(hidden_state)\n",
    "        up, gate = out.chunk(2, -1)\n",
    "        return up * self.act_fn(gate)\n",
    "\n",
    "class TritonFusedMulSiLU(torch.nn.Module):\n",
    "    def __init__(self, hidden_size, intermediate_size=None):\n",
    "        super().__init__()\n",
    "        if intermediate_size is None:\n",
    "            intermediate_size = hidden_size * 8\n",
    "        self.w = torch.nn.Linear(hidden_size, intermediate_size, bias=False)\n",
    "\n",
    "    def forward(self, hidden_state):\n",
    "        return fused_mul_silu(hidden_state, self.w.weight.T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "bs = 32\n",
    "hidden_size = 128\n",
    "dtype = torch.float32\n",
    "device = 'cuda'\n",
    "x1 = torch.randn(bs, hidden_size).to(device).to(dtype)\n",
    "x1.requires_grad_(True)\n",
    "x2 = deepcopy(x1)\n",
    "dy = torch.ones(bs, hidden_size*4).to(device).to(dtype)\n",
    "up = torch.zeros_like(dy)\n",
    "gate = torch.zeros_like(dy)\n",
    "up.requires_grad_(True)\n",
    "gate.requires_grad_(True)\n",
    "torch_mul_silu = MulSiLU(hidden_size).to(device).to(dtype)\n",
    "y1 = torch_mul_silu(x1)\n",
    "y2 = fused_mul_silu(x2, torch_mul_silu.w.weight.data.T)\n",
    "print(torch.allclose(y1, y2, atol=1e-2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if x2.grad is not None:\n",
    "    x2.grad.zero_()\n",
    "if x1.grad is not None:\n",
    "    x1.grad.zero_()\n",
    "y1 = torch_mul_silu(x1)\n",
    "_y2 = fused_mul_silu(x2, torch_mul_silu.w.weight.data.T)\n",
    "y1.backward(dy)\n",
    "_y2.backward(dy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.6057, -0.8023,  0.2951,  ..., -1.0009,  0.3365, -0.4764],\n",
       "        [-0.2243,  0.6240,  0.1433,  ..., -0.1300, -0.2523, -0.8741],\n",
       "        [-0.5612,  0.3279, -0.0631,  ...,  0.0737,  0.6068,  1.0419],\n",
       "        ...,\n",
       "        [ 1.1057,  0.7209,  1.0173,  ...,  0.0222, -1.0308,  0.8422],\n",
       "        [ 0.7728, -0.7765, -0.0887,  ..., -0.1251, -0.4254, -0.4571],\n",
       "        [-0.1539,  0.5414, -0.2653,  ..., -0.7188, -0.0299,  0.9419]],\n",
       "       device='cuda:0')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x1.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.6050, -0.8010,  0.2944,  ..., -0.9995,  0.3357, -0.4756],\n",
       "        [-0.2238,  0.6233,  0.1426,  ..., -0.1295, -0.2516, -0.8726],\n",
       "        [-0.5606,  0.3274, -0.0631,  ...,  0.0737,  0.6061,  1.0404],\n",
       "        ...,\n",
       "        [ 1.1045,  0.7201,  1.0159,  ...,  0.0227, -1.0286,  0.8411],\n",
       "        [ 0.7715, -0.7755, -0.0883,  ..., -0.1245, -0.4249, -0.4564],\n",
       "        [-0.1535,  0.5406, -0.2647,  ..., -0.7173, -0.0295,  0.9404]],\n",
       "       device='cuda:0')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x2.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bs: 8, seq_len: 1024\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlwAAAGwCAYAAAB8crvUAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/GU6VOAAAACXBIWXMAAA9hAAAPYQGoP6dpAABidUlEQVR4nO3deVxUVf8H8M+wDSAMKDsKIpLgvktklimJRj6aWqbmvmRhuZSaT6W26k9bNNcyE8vMpdTMBSUUfEzURDHccEOxZHGDYZF1zu+PG1dGUBGBO8x83q/XvJy598yd7xVjPp177jkqIYQAEREREVUbM6ULICIiIjJ2DFxERERE1YyBi4iIiKiaMXARERERVTMGLiIiIqJqxsBFREREVM0YuIiIiIiqmYXSBRgLnU6Hq1evwt7eHiqVSulyiIiIqAKEEMjKyoKnpyfMzKqvH4qBq4pcvXoVXl5eSpdBRERElXDlyhU0aNCg2o7PwFVF7O3tAUg/MI1Go3A1REREVBFarRZeXl7y93h1YeCqIiWXETUaDQMXERFRLVPdw4E4aJ6IiIiomjFwEREREVUzBi4iIiKiasYxXDWsuLgYhYWFSpdBVG0sLS1hbm6udBlERAaFgauGCCGQmpqKjIwMpUshqnaOjo5wd3fnnHRERP9i4KohJWHL1dUVtra2/CIioySEQG5uLtLT0wEAHh4eCldERGQYGLhqQHFxsRy2nJyclC6HqFrZ2NgAANLT0+Hq6srLi0RE4KD5GlEyZsvW1lbhSohqRsm/dY5XJCKSMHDVIF5GJFPBf+tERPoYuIiIiIiqGQMXERERUTVj4KIqNXv2bLRp00bpMgxGeHg4HB0dq/UzunbtikmTJlW4/aVLl6BSqRAfH19tNRERkT4GLronlUp138fs2bPLvOftt99GVFSU/HrEiBHo27dvjdXs4+NTps4GDRrU2Oc/SHh4OFQqFZo2bVpm38aNG6FSqeDj4/NIn5GUlITBgwfD09MT1tbWaNCgAfr06YMzZ84AALy8vJCSkoIWLVoAuH8Ai46OhkqlKnf+OB8fHyxYsOCRaiUiqqht2wCdTukqKo/TQtA9paSkyM/Xr1+PmTNnIjExUd5mZ2cnPxdCoLi4GHZ2dnrblfDhhx9i7Nix8mtDm5agTp06SE9PR2xsLIKCguTtK1euhLe39yMdu7CwEM8++yz8/f2xadMmeHh44O+//8bOnTvl0GRubg53d/dH+hwiopq0ciUwZgzQpw+waRNgVgu7i2phycZBCCAnp+YfQlS8Rnd3d/nh4OAAlUolvz5z5gzs7e2xc+dOtG/fHmq1Gvv379e7pDh79mysXr0av/76q9zbFB0dDQBISEhAt27dYGNjAycnJ4wbNw7Z2dnyZ5f0jH322Wfw8PCAk5MTwsLCKjTNgL29vV7tLi4uAMrvkWnTpo3cUyeEwOzZs+Ht7Q21Wg1PT0+8+eabctv8/Hy8/fbbqF+/PurUqYPAwED5fEqEh4fD29sbtra2eOGFF3Djxo0y9VlYWGDw4MH47rvv5G1///03oqOjMXjwYL225fUQTpo0CV27di333E+ePIkLFy5g6dKlePzxx9GwYUN07twZH3/8MR5//HEAvKRIRLXLsWNAWJj0vFOn2hm2APZwKSY3F1CiIyg7G6hTp+qO98477+Czzz6Dr68v6tatqxdA3n77bZw+fRparRarVq0CANSrVw85OTkICQlBUFAQ/vzzT6Snp2PMmDGYMGECwsPD5ffv3bsXHh4e2Lt3L86fP4+BAweiTZs2er1XVemXX37Bl19+iXXr1qF58+ZITU3F8ePH5f0TJkzAqVOnsG7dOnh6emLz5s3o2bMnEhIS8Nhjj+HQoUMYPXo05syZg759+yIiIgKzZs0q97NGjRqFrl27YuHChbC1tUV4eDh69uwJNze3RzoHFxcXmJmZ4eeff8akSZMMrnePiOhh3LoFDBgA5OcDzz8PvPOO0hVVXi3NiWQoPvzwQzz77LNo3Lgx6tWrp7fPzs4ONjY2UKvVcm+TlZUV1q5di7y8PHz//fdo0aIFunXrhsWLF+OHH35AWlqa/P66deti8eLFCAgIwPPPP4/Q0FC98WH3Mn36dPnSpp2dHb766qsKnUtycjLc3d0RHBwMb29vdOrUSQ53ycnJWLVqFTZu3IguXbqgcePGePvtt/Hkk0/KYXLhwoXo2bMnpk2bhiZNmuDNN99ESEhIuZ/Vtm1b+Pr64ueff4YQAuHh4Rg1alSF6ryf+vXr46uvvsLMmTNRt25ddOvWDR999BEuXrz4yMcmIqpJOh0wfDhw8SLQqBHw/fe1t3cLUDhwLVu2DK1atYJGo4FGo0FQUBB27twp78/Ly0NYWBicnJxgZ2eH/v37630hA9IXYWhoKGxtbeHq6oqpU6eiqKhIr010dDTatWsHtVoNPz8/vV6UEkuWLIGPjw+sra0RGBiIw4cPV8s5l7C1lXqbavpR1ZPdd+jQ4aHfc/r0abRu3Rp1SnW1de7cGTqdTm+MWPPmzfV6aDw8POQ1+j799FO9UJWcnCy3mzp1KuLj4+XHsGHDKlTXiy++iNu3b8PX1xdjx47F5s2b5X9LCQkJKC4uRpMmTfQ+NyYmBhcuXJDPKzAwUO+Ypcdo3W3UqFFYtWoVYmJikJOTg+eee65CdT5IWFgYUlNT8eOPPyIoKAgbN25E8+bNERkZWSXHJyKqCfPmAb/9BqjVwM8/A3XrKl3Ro1H0kmKDBg0wd+5cPPbYYxBCYPXq1ejTpw+OHTuG5s2bY/Lkydi+fTs2btwIBwcHTJgwAf369cMff/wBQFqjMDQ0FO7u7jhw4ABSUlIwbNgwWFpa4tNPPwUg3bEVGhqK8ePH48cff0RUVBTGjBkDDw8Pufdh/fr1mDJlCpYvX47AwEAsWLAAISEhSExMhKura7Wcu0pVtZf2lFKnGk/C0tJS77VKpYLu31tUxo8fj5deekne5+npKT93dnaGn59fmeOZmZlB3DWIrfSYMC8vLyQmJuL3339HZGQkXn/9dcyfPx8xMTHIzs6Gubk54uLiylymq+xNAkOGDMG0adMwe/ZsDB06FBYWZf9zfFDN92Jvb4/evXujd+/e+PjjjxESEoKPP/4Yzz777EPVqNFoAACZmZllprfIyMiAg4PDQx2PiKgi9u4F3n1Xer5oEdCunbL1VAVFe7h69+6N5557Do899hiaNGmCTz75BHZ2djh48CAyMzOxcuVKfPHFF+jWrRvat2+PVatW4cCBAzh48CAAYPfu3Th16hTWrFmDNm3aoFevXvjoo4+wZMkSFBQUAACWL1+ORo0a4fPPP0fTpk0xYcIEDBgwAF9++aVcxxdffIGxY8di5MiRaNasGZYvXw5bW1u9Qc1UOVZWViguLtbb1rRpUxw/fhw5OTnytj/++ANmZmbw9/ev0HHr1asHPz8/+VFeWLmbi4uL3p2XWq0WSUlJem1sbGzQu3dvfPXVV4iOjkZsbCwSEhLQtm1bFBcXIz09Xe9z/fz85Dv+mjZtikOHDukdr+Tf6r3O4T//+Q9iYmLueTnx7poBPPRgd5VKhYCAAL2/74p67LHHYGZmhri4OL3tFy9eRGZmJpo0afLQxyQiup9//gFeflm6pDhihHR3ojEwmKuhxcXFWLduHXJychAUFIS4uDgUFhYiODhYbhMQEABvb2/ExsYCAGJjY9GyZUu9gcYhISHQarU4efKk3Kb0MUralByjoKAAcXFxem3MzMwQHBwstylPfn4+tFqt3oPK8vHxwV9//YXExERcv34dhYWFGDJkCKytrTF8+HCcOHECe/fuxRtvvIGhQ4c+8qDx++nWrRt++OEH/O9//0NCQgKGDx+u11sVHh6OlStX4sSJE7h48SLWrFkDGxsbNGzYEE2aNMGQIUMwbNgwbNq0CUlJSTh8+DDmzJmD7du3AwDefPNNRERE4LPPPsO5c+ewePFiRERE3Lem8PBwXL9+HQEBAfes+ciRI/j+++9x7tw5zJo1CydOnLjn8eLj49GnTx/8/PPPOHXqFM6fP4+VK1fiu+++Q58+fe5bS2Jiot6l2Pj4eFhbW2PMmDF46623sHXrViQlJWHfvn0YMmQIHn/8cTzxxBP3PSYR0cMoLAQGDgTS04FWrYAlS6QrQsZA8cCVkJAAOzs7qNVqjB8/Hps3b0azZs2QmpoKKyurMpcx3NzckJqaCgBITU0t8wVd8vpBbbRaLW7fvo3r16+juLi43DYlxyjPnDlz4ODgID+8vLwqdf7GbuzYsfD390eHDh3g4uKCP/74A7a2tti1axdu3ryJjh07YsCAAejevTsWL15crbXMmDEDTz/9tDwAv2/fvmjcuLG839HREStWrEDnzp3RqlUr/P777/jtt9/g5OQEAFi1ahWGDRuGt956C/7+/ujbty/+/PNPee6sxx9/HCtWrMDChQvRunVr7N69G++99959ayqZFuNeQkJC8P7772PatGno2LEjsrKy7jsmrUGDBvDx8cEHH3yAwMBAtGvXDgsXLsQHH3yAd0v65+/h5ZdfRtu2bfUeaWlpWLhwIYYPH47p06ejefPmGDFiBFq1aoXffvuNi1QTUZV65x3gjz8AjQb45ZeqH3esJJW4e4BIDSsoKEBycjIyMzPx888/49tvv0VMTAzi4+MxcuRI5Ofn67Xv1KkTnnnmGfzf//0fxo0bh8uXL2PXrl3y/tzcXNSpUwc7duxAr1690KRJE4wcORIzZsyQ2+zYsQOhoaHIzc3FrVu3UL9+fRw4cEBvgPO0adMQExNT5hJRifz8fL3atFotvLy8kJmZKY97KZGXl4ekpCQ0atQI1tbWj/T3RVQb8N88ET2sn38GXnxRer55M1BTi5RotVo4ODiU+/1dlRSfh8vKykoe4Ny+fXv8+eefWLhwIQYOHIiCggJkZGTo9XKlpaXJY2bc3d3L3E1Ychdj6TZ339mYlpYGjUYDGxsbmJubw9zcvNw295uNW61WQ61WV+6kiYiISHb2LFAylHXq1JoLWzVJ8UuKd9PpdMjPz0f79u1haWmpN+9SYmIikpOT5Z6ooKAgJCQkyFMFAEBkZCQ0Gg2aNWsmt7l77qbIyEj5GFZWVmjfvr1eG51Oh6ioqPve0k9ERESPLicH6N8fyMoCnnoK+HeSAaOjaA/XjBkz0KtXL3h7eyMrKwtr165FdHQ0du3aBQcHB4wePRpTpkxBvXr1oNFo8MYbbyAoKEheoqRHjx5o1qwZhg4dinnz5iE1NRXvvfcewsLC5N6n8ePHY/HixZg2bRpGjRqFPXv2YMOGDfJAZwCYMmUKhg8fjg4dOqBTp05YsGABcnJyMHLkSEX+XoiIiEyBEMD48cCJE4C7O7BuHVCBm85rJUVPKz09HcOGDUNKSgocHBzQqlUr7Nq1S54r6Msvv4SZmRn69++P/Px8hISEYOnSpfL7zc3NsW3bNrz22msICgpCnTp1MHz4cHz44Ydym0aNGmH79u2YPHkyFi5ciAYNGuDbb7/VmwF84MCBuHbtGmbOnInU1FS0adMGERER1XrHHBERkan7+mtgzRrA3BxYvx7w8FC6ouqj+KB5Y3G/QXccQEymhv/miehBjhwBOncGCgqkWeWnTlWmjpoaNG9wY7iIiIjIuN24IS1KXVAgDZB/+22lK6p+DFxERERUY3Q6YOhQ4PJlwM8PCA83nslN74eBi4iIiGrMJ58AO3cC1tbS3FumsiQrAxcZpK5du2LSpElKl1EtwsPDy6yg8CDG/PdBRKYjMhKYNUt6vmwZ0Lq1svXUJAYuuq8RI0agbzkz0EVHR0OlUiEjI0N+fvfjfsvaVCZ0KKXkfO5eiDo/Px9OTk5QqVSIjo6u9PGLi4sxd+5cBAQEwMbGBvXq1UNgYCC+/fZbuc2mTZvw0Ucfya/vF8B8fHywYMGCMttnz56NNm3aVLpOIqJHceUKMGiQNBXE2LHSwtSmxEhnuyAlJCYm6t3hYWdnp2A1VcvLywurVq2S54ADgM2bN8POzg43b958pGN/8MEH+Prrr7F48WJ06NABWq0WR44cwa1bt+Q29erVe6TPICJSUkGBtGzPjRtAu3bAV18pXVHNYw8XVRlXV1e4u7vLj6oMXD/88AM6dOgAe3t7uLu7Y/DgwXorDJT0su3atQtt27aFjY0NunXrhvT0dOzcuRNNmzaFRqPB4MGDkZubK78vIiICTz75JBwdHeHk5ITnn38eFy5cKPP5w4cPx7p163D79m1523fffYfhw4frtSvd81ciPj4eKpUKly5dKvfctm7ditdffx0vvvgiGjVqhNatW2P06NF4u9RtO7ykSES12dtvA4cOAY6O0rgtU5wthoFLIUII5BTk1Pijtk67VlhYiI8++gjHjx/Hli1bcOnSJYwopz969uzZWLx4MQ4cOIArV67gpZdewoIFC7B27Vps374du3fvxqJFi+T2OTk5mDJlCo4cOYKoqCiYmZnhhRdegE6n0ztu+/bt4ePjg19++QUAkJycjH379mHo0KGPfG7u7u7Ys2cPrl279sjHIiIyNOvWASW/dn/4AWjUSNl6lMJLigrJLcyF3Zyav+SWPSMbdazqPNR7tm3bVqa3qri4uEy7Bg0a6L2+fPkynJycHr7IcowqWdUUgK+vL7766it07NgR2dnZerV9/PHH6Ny5MwBg9OjRmDFjBi5cuABfX18AwIABA7B3715Mnz4dANC/f3+9z/nuu+/g4uKCU6dOoUWLFmVq+O677/DKK68gPDwczz33HFxcXB753L744gsMGDAA7u7uaN68OZ544gn06dMHvXr1euRjExEp6dQpYMwY6fl//ws8/7yy9SiJPVz0QM888wzi4+P1HqUHdJf43//+p9embt26AKSxXCWP8ePHV6qGuLg49O7dG97e3rC3t8fTTz8NQOppKq1Vq1byczc3N9ja2sphq2Rb6UuR586dw6BBg+Dr6wuNRgMfH59yjwsAr7zyCmJjY3Hx4kWEh4frhcBH0axZM5w4cQIHDx7EqFGjkJ6ejt69e2NMyW8pIqJaKCtLWpQ6Jwfo1g0oteqeSWIPl0JsLW2RPSNbkc99WHXq1IGfn5/etr///rtMu0aNGpV752F8fLz8vDLLJuTk5CAkJAQhISH48ccf4eLiguTkZISEhKCgoECvraWlpfxcpVLpvS7ZVvpyYe/evdGwYUOsWLECnp6e0Ol0aNGiRZnjApDHeI0ePRp5eXno1asXsrKy9NqYmUn/D1P60m1hYeEDz9HMzAwdO3ZEx44dMWnSJKxZswZDhw7Fu+++i0YP2f+u0WiQmZlZZntGRgYcTGXCGyJSVMmdiGfOAJ6ewE8/SeslmjIGLoWoVKqHvrRXW90d1h7WmTNncOPGDcydOxdeXl4AgCNHjjxyXTdu3EBiYiJWrFiBLl26AAD2799/3/eMGjUKzz33HKZPnw7zcn57lFxiTElJkXv4SgfOimrWrBkAKWw+LH9/f8TFxZXZfvToUfj7+z/08YiIHtbixdJi1BYWwIYNgKur0hUpj4GLFFNcXFwmjKjVajRt2lRvm7e3N6ysrLBo0SKMHz8eJ06c0JuTqrLq1q0LJycnfPPNN/Dw8EBycjLeeeed+76nZ8+euHbt2j176vz8/ODl5YXZs2fjk08+wdmzZ/H555/f95gDBgxA586d8cQTT8Dd3R1JSUmYMWMGmjRpgoCAgHu+79q1a2X+/jw8PDB58mR06dIFn3zyCfr164fi4mL89NNPiI2NxdKlS+9bCxHRozp4EHjrLen5/PnSAtXEMVykoOzsbLRt21bv0bt37zLtXFxcEB4ejo0bN6JZs2aYO3cuPvvss0f+fDMzM6xbtw5xcXFo0aIFJk+ejPnz59/3PSqVCs7OzrCysip3v6WlJX766SecOXMGrVq1wv/93//h448/vu8xQ0JC8Ntvv6F3795o0qQJhg8fjoCAAOzevRsWFvf+f6K1a9eW+ftbsWIFnnjiCezcuRM7d+5E586d0bVrVxw4cABRUVFlbgQgIqpK165J820VFkp/TpyodEWGQyVq6zwBBkar1cLBwQGZmZllej/y8vKQlJSERo0awdoUJx8hk8N/80Smp7gY6NkT+P13wN8fOHwYqMSw3Rp3v+/vqsQeLiIiInpkH3wghS1bW+CXX2pH2KpJDFxERET0SHbsAEqG1n7zDdC8ubL1GCIGLiIiIqq0S5eAV16Rnr/+OjBkiKLlGCwGLiIiIqqU/HxpcPytW0CnTsAXXyhdkeFi4KpBvD+BTAX/rROZhkmTgCNHgHr1pPm21GqlKzJcDFw1oGS289zcXIUrIaoZJf/W757pn4iMxw8/AMuXAyoV8OOPQMOGSldk2DjxaQ0wNzeHo6OjvIafra0tVCqVwlURVT0hBHJzc5Geng5HR8dyZ+MnotovIQF49VXp+cyZ0nQQdH8MXDXE3d0dAPQWTiYyVo6OjvK/eSIyLlqttCj17dtAjx7A++8rXVHtwMBVQ1QqFTw8PODq6lqhxYyJaitLS0v2bBEZKSGAkSOBc+cALy/pUiL/c68YBq4aZm5uzi8jIiKqlb78Eti0CbC0BDZuBJydla6o9uCgeSIiInqg/fuBadOk519+CQQGKltPbcPARURERPeVlga89JK0XuKgQdIEp/RwGLiIiIjonoqKgJdfBlJSgGbNpKV7eKP9w2PgIiIiont6/30gOhqws5MWpbazU7qi2omBi4iIiMq1dSswd670fOVKICBA2XpqMwYuIiIiKuPCBWDYMOn5xInSGC6qPAYuIiIi0nP7NjBgAJCZCQQFAfPmKV1R7cfARURERHreeAOIj5fm2dqwAbCyUrqi2o+Bi4iIiGTffSeN11KpgJ9+Aho0ULoi48DARURERACkXq2wMOn5Rx8BwcGKlmNUGLiIiIgIGRnSotR5eUBoKDBjhtIVGRcGLiIiIhOn0wHDhwMXLwI+PsD33wNmTAhVin+dREREJm7+fGnOLSsr4OefgXr1lK7I+DBwERERmbDoaOC//5WeL1oEtG+vaDlGi4GLiIjIRF29Kq2TqNNJk5yOHat0RcaLgYuIiMgEFRYCAwcCaWlAy5bAsmVclLo6MXARERGZoBkzgP37AY1GWpTa1lbpiowbAxcREZGJ2bQJ+Pxz6fmqVcBjjylbjylg4CIiIjIhZ88CI0ZIz996C+jXT9FyTIaigWvOnDno2LEj7O3t4erqir59+yIxMVGvTdeuXaFSqfQe48eP12uTnJyM0NBQ2NrawtXVFVOnTkVRUZFem+joaLRr1w5qtRp+fn4IDw8vU8+SJUvg4+MDa2trBAYG4vDhw1V+zkRERErJzZUWpc7KArp0AebMUboi06Fo4IqJiUFYWBgOHjyIyMhIFBYWokePHsjJydFrN3bsWKSkpMiPeaWWLS8uLkZoaCgKCgpw4MABrF69GuHh4Zg5c6bcJikpCaGhoXjmmWcQHx+PSZMmYcyYMdi1a5fcZv369ZgyZQpmzZqFo0ePonXr1ggJCUF6enr1/0UQERFVMyGA8eOBhATAzQ1Yvx6wtFS6KhMiDEh6eroAIGJiYuRtTz/9tJg4ceI937Njxw5hZmYmUlNT5W3Lli0TGo1G5OfnCyGEmDZtmmjevLne+wYOHChCQkLk1506dRJhYWHy6+LiYuHp6SnmzJlTodozMzMFAJGZmVmh9kRERDVp+XIhACHMzITYu1fpagxHTX1/G9QYrszMTABAvbumuP3xxx/h7OyMFi1aYMaMGcjNzZX3xcbGomXLlnBzc5O3hYSEQKvV4uTJk3Kb4LtW4AwJCUFsbCwAoKCgAHFxcXptzMzMEBwcLLe5W35+PrRard6DiIjIEB05Arz5pvR8zhyga1dFyzFJFkoXUEKn02HSpEno3LkzWrRoIW8fPHgwGjZsCE9PT/z111+YPn06EhMTsWnTJgBAamqqXtgCIL9OTU29bxutVovbt2/j1q1bKC4uLrfNmTNnyq13zpw5+OCDDx7tpImIiKrZzZvSuK2CAqBPH2DqVKUrMk0GE7jCwsJw4sQJ7N+/X2/7uHHj5OctW7aEh4cHunfvjgsXLqBx48Y1XaZsxowZmDJlivxaq9XCy8tLsXqIiIjuptMBr7wCXL4MNG4MhIdzclOlGETgmjBhArZt24Z9+/ahQYMG920bGBgIADh//jwaN24Md3f3MncTpqWlAQDc3d3lP0u2lW6j0WhgY2MDc3NzmJubl9um5Bh3U6vVUKvVFT9JIiKiGvbpp8DOnYC1tbQotaOj0hWZLkXHcAkhMGHCBGzevBl79uxBo0aNHvie+Ph4AICHhwcAICgoCAkJCXp3E0ZGRkKj0aBZs2Zym6ioKL3jREZGIigoCABgZWWF9u3b67XR6XSIioqS2xAREdUmv/8OlNywv3Qp0KaNouVQtQ7Jf4DXXntNODg4iOjoaJGSkiI/cnNzhRBCnD9/Xnz44YfiyJEjIikpSfz666/C19dXPPXUU/IxioqKRIsWLUSPHj1EfHy8iIiIEC4uLmLGjBlym4sXLwpbW1sxdepUcfr0abFkyRJhbm4uIiIi5Dbr1q0TarVahIeHi1OnTolx48YJR0dHvbsf74d3KRIRkaG4ckUIZ2fprsTRo5WuxrDV1Pe3ooELQLmPVatWCSGESE5OFk899ZSoV6+eUKvVws/PT0ydOrXMX8qlS5dEr169hI2NjXB2dhZvvfWWKCws1Guzd+9e0aZNG2FlZSV8fX3lzyht0aJFwtvbW1hZWYlOnTqJgwcPVvhcGLiIiMgQ5OcL8fjjUthq21aIf/sw6B5q6vtbJYQQSvWuGROtVgsHBwdkZmZCo9EoXQ4REZmoiROBr76SxmvFxQG+vkpXZNhq6vvboObhIiIiospbv14KWwDw/fcMW4aEgYuIiMgInD4NjB4tPX/nHaB3b2XrIX0MXERERLVcdjbQvz+QkwM88wzw0UdKV0R3Y+AiIiKqxYQAxo2Terg8PICffgIsDGKWTSqNgYuIiKgWW7JEClnm5sCGDcBdq9SRgWDgIiIiqqUOHgRKVpmbPx948kll66F7Y+AiIiKqha5fB156CSgslBannjRJ6Yrofhi4iIiIapniYmDIEODKFaBJE2DlSi5KbegYuIiIiGqZDz8Edu8GbG2BX34BON+24WPgIiIiqkUiIu5M+/D110CLFsrWQxXDwEVERFRLXL4sXUoUAhg/HnjlFaUroopi4CIiIqoF8vOlwfE3bwIdOgALFihdET0MBi4iIqJaYPJk4MgRoF49YONGQK1WuiJ6GAxcREREBm7NGmDZMulOxDVrAB8fpSuih8XARUREZMCio+8sSv3ee0CvXoqWQ5XEwEVERGSg4uOBPn2AggLghReAWbOUrogqi4GLiIjIAF24APTsCWi1wNNPA2vXSuslUu3EwEVERGRg0tKAkBDpz9atgV9/Baytla6KHgUDFxERkQHRaqVxWhcuAI0aATt3Ag4OSldFj4qBi4iIyEDk5wN9+wLHjgEuLtLyPR4eSldFVYGBi4iIyAAUF0szx+/dC9jZST1bfn5KV0VVhYGLiIhIYUIAb7wB/PwzYGUFbNkCtG+vdFVUlRi4iIiIFPbhh/oTm3bvrnRFVNUYuIiIiBS0fDkwe7b0fPFi4MUXFS2HqgkDFxERkUJ+/hl4/XXp+cyZd56T8WHgIiIiUsDevcCQIdL4rXHj7vRykXFi4CIiIqphx47dWbKnXz9g6VJp/BYZLwYuIiKiGnThgjSxaVYW0LUr8OOPXLLHFDBwERER1ZDUVKBHjztL9mzZwiV7TAUDFxERUQ0oWbLn4kVpyZ6ICC7ZY0oYuIiIiKpZXp60ZE98PODqKi3Z4+6udFVUkxi4iIiIqlHpJXvs7blkj6li4CIiIqomQgATJgC//HJnyZ527ZSuipTAwEVERFRNPvhAmklepZLuRuzWTemKSCkMXERERNVg2TIpcAHAkiXAgAHK1kPKYuAiIiKqYj//DISFSc9nzgRee03Zekh5DFxERERVaM+eO0v2vPoql+whCQMXERFRFTl2TJr+oaAA6N9fupTIJXsIYOAiIiKqEufPAz173lmyZ80aLtlDdzBwERERPaLUVCAkBEhPB9q04ZI9VBYDFxER0SPIzLyzZI+vrzSxKZfsobsxcBEREVUSl+yhimLgIiIiqoTiYuluxOhoacmeiAigcWOlqyJDxcBFRET0kISQ5tnatOnOkj1t2ypdFRkyRQPXnDlz0LFjR9jb28PV1RV9+/ZFYmKiXpu8vDyEhYXByckJdnZ26N+/P9LS0vTaJCcnIzQ0FLa2tnB1dcXUqVNRVFSk1yY6Ohrt2rWDWq2Gn58fwsPDy9SzZMkS+Pj4wNraGoGBgTh8+HCVnzMREdV+H3wAfP01l+yhilM0cMXExCAsLAwHDx5EZGQkCgsL0aNHD+Tk5MhtJk+ejN9++w0bN25ETEwMrl69in79+sn7i4uLERoaioKCAhw4cACrV69GeHg4Zs6cKbdJSkpCaGgonnnmGcTHx2PSpEkYM2YMdu3aJbdZv349pkyZglmzZuHo0aNo3bo1QkJCkJ6eXjN/GUREVCtwyR6qFGFA0tPTBQARExMjhBAiIyNDWFpaio0bN8ptTp8+LQCI2NhYIYQQO3bsEGZmZiI1NVVus2zZMqHRaER+fr4QQohp06aJ5s2b633WwIEDRUhIiPy6U6dOIiwsTH5dXFwsPD09xZw5cypUe2ZmpgAgMjMzH/KsiYiottiwQQiVSghAiFmzlK6GqkJNfX8b1BiuzMxMAEC9evUAAHFxcSgsLERwcLDcJiAgAN7e3oiNjQUAxMbGomXLlnBzc5PbhISEQKvV4uTJk3Kb0scoaVNyjIKCAsTFxem1MTMzQ3BwsNzmbvn5+dBqtXoPIiIyXnv2AK+8Io3fGj8emDVL6YqoNjGYwKXT6TBp0iR07twZLVq0AACkpqbCysoKjo6Oem3d3NyQmpoqtykdtkr2l+y7XxutVovbt2/j+vXrKC4uLrdNyTHuNmfOHDg4OMgPLy+vyp04EREZvKNHgT597izZs3gxl+yhh2MwgSssLAwnTpzAunXrlC6lQmbMmIHMzEz5ceXKFaVLIiKianD+vDSxaXY2l+yhyrNQugAAmDBhArZt24Z9+/ahQYMG8nZ3d3cUFBQgIyNDr5crLS0N7v/OLOfu7l7mbsKSuxhLt7n7zsa0tDRoNBrY2NjA3Nwc5ubm5bZxv8cMdmq1Gmq1unInTEREtcLdS/b8+iuX7KHKUbSHSwiBCRMmYPPmzdizZw8aNWqkt799+/awtLREVFSUvC0xMRHJyckICgoCAAQFBSEhIUHvbsLIyEhoNBo0a9ZMblP6GCVtSo5hZWWF9u3b67XR6XSIioqS2xARkWnJzJQWoy69ZI9Go3RVVGtV65D8B3jttdeEg4ODiI6OFikpKfIjNzdXbjN+/Hjh7e0t9uzZI44cOSKCgoJEUFCQvL+oqEi0aNFC9OjRQ8THx4uIiAjh4uIiZsyYIbe5ePGisLW1FVOnThWnT58WS5YsEebm5iIiIkJus27dOqFWq0V4eLg4deqUGDdunHB0dNS7+/F+eJciEZHxuH1biKeflu5GdHMT4vx5pSui6lJT39+KBi4A5T5WrVolt7l9+7Z4/fXXRd26dYWtra144YUXREpKit5xLl26JHr16iVsbGyEs7OzeOutt0RhYaFem71794o2bdoIKysr4evrq/cZJRYtWiS8vb2FlZWV6NSpkzh48GCFz4WBi4jIOBQVCdGvnxS27O2FOHpU6YqoOtXU97dKCCGU6l0zJlqtFg4ODsjMzISGfc5ERLVSyZQP33wjLdkTEQE884zSVVF1qqnvb4O5S5GIiEhps2dLYatkyR6GLaoqDFxEREQAli4FPvzwznMu2UNViYGLiIhM3oYNwIQJ0vPZs6XLikRViYGLiIhMWlTUnSV7XnsNmDlT6YrIGDFwERGRyTp6FOjbFygslC4hLlrEJXuoejBwERGRSSq9ZM8zz3DJHqpeDFxERGRyUlKAHj2kJXvatgW2bAG4WhtVJwYuIiIyKZmZUs9WUhLQuDGX7KGawcBFREQmIy8P6NMHOH4ccHMDdu2S/iSqbgxcRERkEoqLgcGDgZgYwN5e6tlq3FjpqshUMHAREZHREwJ4/XVg82ZpyZ5ff5XGbhHVFAYuIiIyerNm3VmyZ+1aLtlDNY+Bi4iIjNrixcBHH0nPly0D+vdXth4yTQxcRERktDZsAN58U3r+wQfAq68qWw+ZLgYuIiIySr//fmfJntdfB95/X+mKyJQxcBERkdGJiwNeeOHOkj1ffcUle0hZDFxERGRUzp27s2RPt25csocMAwMXEREZjZIle65dA9q1k6aB4JI9ZAgYuIiIyChkZAA9ewKXLkkTmu7YwSV7yHAwcBERUa1XsmTPX39JS/Xs3s0le8iwMHAREVGtVrJkz759Uo9WRATg66t0VUT6GLiIiKjWEgJ47bU7Y7V+/RVo00bpqojKYuAiIqJaa+ZMYMUKwMxMWrKna1elKyIqHwMXERHVSosWAR9/LD1fuhTo10/Zeojuh4GLiIhqnfXrgYkTpedcsodqAwYuIiKqVX7/HRg6VBq/FRbGJXuodmDgIiKiWuPIkTtL9rz4IrBwIZfsodqBgYuIiGqFc+eA556Tluzp3h344Qcu2UO1BwMXEREZvKtX9Zfs2bSJS/ZQ7cLARUREBu3mTWkxai7ZQ7UZAxcRERmstDRpbq2//gLc3blkD9VeFkoXQEREVJ6//waCg4HERCls/f47l+yh2ouBi4iIDE5SkjQwPikJ8PICoqKAxx5TuiqiyuMlRSIiMiiJiUCXLlLYatwY+N//GLao9mPgIiIig5GQADz1FPDPP0DTpsC+fUDDhkpXRfToGLiIiMggHDkiDZBPTwfatAFiYgBPT6WrIqoaDFxERKS4P/6QxmzdvAkEBgJ79gAuLkpXRVR1GLiIiEhRUVHSpKZarXQ5MTISqFtX6aqIqlalAtfq1auxfft2+fW0adPg6OiIJ554ApcvX66y4oiIyLht3w6EhgK5uVLo2rkTsLdXuiqiqlepwPXpp5/CxsYGABAbG4slS5Zg3rx5cHZ2xuTJk6u0QCIiMk6//CItRJ2fD/TpA2zdCtjaKl0VUfWo1DxcV65cgZ+fHwBgy5Yt6N+/P8aNG4fOnTuja9euVVkfEREZoTVrgOHDAZ0OePll4PvvAUtLpasiqj6V6uGys7PDjRs3AAC7d+/Gs88+CwCwtrbG7du3q646IiIyOt98AwwbJoWtkSOl8MWwRcauUj1czz77LMaMGYO2bdvi7NmzeO655wAAJ0+eRENOmEJERPewYAFQMvIkLAz46ivAjLdvkQmo1D/zJUuWICgoCNeuXcMvv/wCJycnAEBcXBwGDx5cpQUSEZFx+OSTO2Fr2jRg0SKGLTIdlfqn7ujoiM8++wzvvvsuioqKsHXrVmzduhXt27dHy5YtK3ycffv2oXfv3vD09IRKpcKWLVv09o8YMQIqlUrv0bNnT702N2/exJAhQ6DRaODo6IjRo0cjOztbr81ff/2FLl26wNraGl5eXpg3b16ZWjZu3IiAgABYW1ujZcuW2LFjR8X/QoiI6J6EAN59F3jvPen1Bx8Ac+cCKpWydRHVpEpdUoyIiMCwYcNw48YNCCH09qlUKhQXF1foODk5OWjdujVGjRqFfv36ldumZ8+eWLVqlfxarVbr7R8yZAhSUlIQGRmJwsJCjBw5EuPGjcPatWsBAFqtFj169EBwcDCWL1+OhIQEjBo1Co6Ojhg3bhwA4MCBAxg0aBDmzJmD559/HmvXrkXfvn1x9OhRtGjRosJ/L0REpE8IqVdr4ULp9fz5wNtvK1sTkSJEJfj5+YnXX39dpKamVubt5QIgNm/erLdt+PDhok+fPvd8z6lTpwQA8eeff8rbdu7cKVQqlfjnn3+EEEIsXbpU1K1bV+Tn58ttpk+fLvz9/eXXL730kggNDdU7dmBgoHj11VcrXH9mZqYAIDIzMyv8HiIiY1ZUJMTYsUJIsUuIJUuUroiorJr6/q7UJcW0tDRMmTIFbm5uVZn9yhUdHQ1XV1f4+/vjtddek++OBKQ5wBwdHdGhQwd5W3BwMMzMzHDo0CG5zVNPPQUrKyu5TUhICBITE3Hr1i25TXBwsN7nhoSEIDY29p515efnQ6vV6j2IiEhSVCRN+7BihTROa9Uq4PXXla6KSDmVClwDBgxAdHR0FZdSVs+ePfH9998jKioK//d//4eYmBj06tVLvmSZmpoKV1dXvfdYWFigXr16SE1NldvcHQxLXj+oTcn+8syZMwcODg7yw8vL69FOlojISBQUSHNr/fgjYGEBrF0LjBihdFVEyqrUGK7FixfjxRdfxP/+9z+0bNkSlndNoPLmm29WSXEvv/yy/Lxly5Zo1aoVGjdujOjoaHTv3r1KPqOyZsyYgSlTpsivtVotQxcRmbzbt4EBA4AdOwArK2DjRuA//1G6KiLlVSpw/fTTT9i9ezesra0RHR0NValbTVQqVZUFrrv5+vrC2dkZ58+fR/fu3eHu7o709HS9NkVFRbh58ybc3d0BAO7u7khLS9NrU/L6QW1K9pdHrVaXGcBPRGTKsrOlJXr27AFsbIAtW6T1EYmokpcU3333XXzwwQfIzMzEpUuXkJSUJD8uXrxY1TXK/v77b9y4cQMeHh4AgKCgIGRkZCAuLk5us2fPHuh0OgQGBspt9u3bh8LCQrlNZGQk/P39Ufff5eiDgoIQFRWl91mRkZEICgqqtnMhIjImmZlASIgUtuzsgIgIhi2i0ioVuAoKCjBw4ECYPeKMddnZ2YiPj0d8fDwAICkpCfHx8UhOTkZ2djamTp2KgwcP4tKlS4iKikKfPn3g5+eHkJAQAEDTpk3Rs2dPjB07FocPH8Yff/yBCRMm4OWXX4anpycAYPDgwbCyssLo0aNx8uRJrF+/HgsXLtS7HDhx4kRERETg888/x5kzZzB79mwcOXIEEyZMeKTzIyIyBTduAN27AwcOAI6OQFQU8NRTSldFZGAqc2vjpEmTxCeffPLIt0ju3btXACjzGD58uMjNzRU9evQQLi4uwtLSUjRs2FCMHTu2zFQUN27cEIMGDRJ2dnZCo9GIkSNHiqysLL02x48fF08++aRQq9Wifv36Yu7cuWVq2bBhg2jSpImwsrISzZs3F9u3b3+oc+G0EERkilJShGjRQpr2wdlZiGPHlK6I6OHU1Pe3Soi7Zi6tgDfffBPff/89WrdujVatWpUZNP/FF188ehKsZbRaLRwcHJCZmQmNRqN0OURE1e7KFSA4GDh7FvDwkHq2mjZVuiqih1NT39+VGjSfkJCAtm3bAgBOnDiht0/FtRqIiIzexYtAt27A5cuAt7cUtvz8lK6KyHBVKnDt3bu3qusgIqJa4swZaczW1atSyIqKkkIXEd0b12knIqIK++svaUD81atAs2bAvn0MW0QVwcBFREQV8uefQNeuwLVrQNu2QEyMNHaLiB6MgYuIiB5o/37pMuKtW8Djj0vzbTk7K10VUe3BwEVERPf1++/SJKZZWVIP1+7d0nxbRFRxDFxERHRP27YBzz8vrZHYs6e0RqK9vdJVEdU+DFxERFSujRuBF14A8vOlP7dskdZIJKKHx8BFRERlfP898PLLQFERMGgQsH49oFYrXRVR7cXARUREepYvB4YPB3Q6YPRo4IcfgLsWFCGih8TARUREsi++AF57TXr+xhvAN98A5ubK1kRkDBi4iIgIQgAffwy89Zb0+p13gIULATN+SxBViUot7UNERMZDCOC//wXmzpVef/QR8O67AJfGJao6DFxERCZMpwMmTQIWLZJef/45MGWKoiURGSUGLiIiE1VcDLz6KrBypfR62TJg/HhlayIyVgxcREQmqLAQGDECWLtWGqe1ahUwbJjSVREZLwYuIiITk58vza21eTNgYSGFrhdfVLoqIuPGwEVEZEJu3wb69QMiIgArK+Dnn4HevZWuisj4MXAREZmIrCzgP/8BoqOlJXp+/RV49lmlqyIyDQxcREQmICMD6NULOHhQWnx6+3agSxelqyIyHQxcRERG7vp1oEcP4NgxoG5dYNcuoGNHpasiMi0MXERERiwlRbpsePIk4OIC/P470KqV0lURmR4GLiIiI5WcDHTvDpw/D3h6AlFRQECA0lURmSYGLiIiI3ThAtCtmxS6fHyksOXrq3RVRKaLy5ISERmZ06elAfHJyUCTJsC+fQxbREpj4CIiMiLHjwNPPy2N3WrRAoiJAby8lK6KiBi4iIiMxKFDQNeuwLVrQLt20nxb7u5KV0VEAAMXEZFR2LcPCA6W5tt64glgzx7AyUnpqoioBAMXEVEtt3s30LMnkJ0tDZTftQtwcFC6KiIqjYGLiKgW27pVWgvx9m3gueeAbdsAOzulqyKiuzFwERHVUuvXA/37AwUF0oLUmzdLayQSkeFh4CIiqoXCw4HBg4GiImDIECl8WVkpXRUR3QsDFxFRLbN0KTByJKDTAWPHAqtXAxacxprIoDFwERHVIp99BoSFSc8nTgS+/howN1e2JiJ6MAYuIqJaQAjgww+BqVOl1//9L/Dll4BKpWxdRFQx7IQmIjJwQgDvvAPMmye9/vhj4N13la2JiB4OAxcRkQHLzZUuIYaHS6+//BKYNEnJioioMhi4iIgM1NmzwIABQEKCdOlw+XJg3DilqyKiyuAYLiIiA7RxI9ChgxS2XF2B339n2CKqzRi4iIgMSH4+8MYbwEsvAVlZwFNPAceOSUv2EFHtxcBFRGQgLl8GunQBFi+WXk+fDkRFAZ6eytZFRI+OY7iIiAzAtm3AsGHArVtA3brA998Dzz+vdFVEVFXYw0VEpKCiImnKh969pbDVsSNw9CjDFpGxYQ8XEZFCrl4FBg0C9u2TXr/xhjSTPNdEJDI+ivZw7du3D71794anpydUKhW2bNmit18IgZkzZ8LDwwM2NjYIDg7GuXPn9NrcvHkTQ4YMgUajgaOjI0aPHo3s7Gy9Nn/99Re6dOkCa2treHl5YV7J7IGlbNy4EQEBAbC2tkbLli2xY8eOKj9fIqISe/YAbdtKYcveXlp8+quvGLaIjJWigSsnJwetW7fGkiVLyt0/b948fPXVV1i+fDkOHTqEOnXqICQkBHl5eXKbIUOG4OTJk4iMjMS2bduwb98+jCt177RWq0WPHj3QsGFDxMXFYf78+Zg9eza++eYbuc2BAwcwaNAgjB49GseOHUPfvn3Rt29fnDhxovpOnohMkk4HfPQR8OyzQHo60LIlcOSIdFciERkxYSAAiM2bN8uvdTqdcHd3F/Pnz5e3ZWRkCLVaLX766SchhBCnTp0SAMSff/4pt9m5c6dQqVTin3/+EUIIsXTpUlG3bl2Rn58vt5k+fbrw9/eXX7/00ksiNDRUr57AwEDx6quvVrj+zMxMAUBkZmZW+D1EZFquXRMiJEQIabEeIUaNEiInR+mqiExbTX1/G+yg+aSkJKSmpiI4OFje5uDggMDAQMTGxgIAYmNj4ejoiA4dOshtgoODYWZmhkOHDsltnnrqKViV6qcPCQlBYmIibt26Jbcp/TklbUo+pzz5+fnQarV6DyKiezlwQLqEuGsXYGMDrFoFrFwJ2NoqXRkR1QSDDVypqakAADc3N73tbm5u8r7U1FS4urrq7bewsEC9evX02pR3jNKfca82JfvLM2fOHDg4OMgPLy+vhz1FIjIBQgBffAE8/TTw999AkybAoUPAiBFKV0ZENclgA5ehmzFjBjIzM+XHlStXlC6JiAxMRgbQvz/w1lvS9A8DB0rjtVq2VLoyIqppBjsthLu7OwAgLS0NHh4e8va0tDS0adNGbpOenq73vqKiIty8eVN+v7u7O9LS0vTalLx+UJuS/eVRq9VQq9WVODMiMgVHjwIvvghcvAhYWgJffgm8/rq0CDURmR6D7eFq1KgR3N3dERUVJW/TarU4dOgQgoKCAABBQUHIyMhAXFyc3GbPnj3Q6XQIDAyU2+zbtw+FhYVym8jISPj7+6Nu3bpym9KfU9Km5HOIiCpKCODrr4EnnpDClo8P8McfQFgYwxaRKVM0cGVnZyM+Ph7x8fEApIHy8fHxSE5OhkqlwqRJk/Dxxx9j69atSEhIwLBhw+Dp6Ym+ffsCAJo2bYqePXti7NixOHz4MP744w9MmDABL7/8Mjz/XXxs8ODBsLKywujRo3Hy5EmsX78eCxcuxJQpU+Q6Jk6ciIiICHz++ec4c+YMZs+ejSNHjmDChAk1/VdCRLVYdjYwdCgwfry0CHXv3lJPV8eOSldGRIqr1nsgH2Dv3r0CQJnH8OHDhRDS1BDvv/++cHNzE2q1WnTv3l0kJibqHePGjRti0KBBws7OTmg0GjFy5EiRlZWl1+b48ePiySefFGq1WtSvX1/MnTu3TC0bNmwQTZo0EVZWVqJ58+Zi+/btD3UunBaCyLSdPClE06bSdA/m5kLMmyeETqd0VUT0IDX1/a0SQggF857R0Gq1cHBwQGZmJjQajdLlEFENWrMGePVVIDcX8PQE1q0DunRRuioiqoia+v422DFcRESGLi9PClpDh0phq3t34Ngxhi0iKouBi4ioEi5cAIKCgG++kQbDz5olTWp619SAREQADHhaCCIiQ7VpEzByJKDVAs7OwI8/Aj16KF0VERky9nAREVVQQQEwebI0malWC3TuDMTHM2wR0YMxcBERVcCVK0DXrsCCBdLrt98G9u4F6tdXsioiqi14SZGI6AEiIoBXXgFu3AAcHIDVq4E+fZSuiohqE/ZwERHdQ3Ex8P77wHPPSWGrfXtpIlOGLSJ6WOzhIiIqR2oqMHiwdNkQkNZB/PxzwNpa2bqIqHZi4CIiuktMDPDyy1LoqlMHWLECGDRI6aqIqDbjJUUion/pdMCcOUC3blLYat4cOHKEYYuIHh17uIiIII3RGjYM2LFDej1sGLB0qdTDRUT0qBi4iMjkHToEvPQSkJwsjdFavBgYNUqaQZ6IqCrwkiIRmSwhgK++ktY+TE4G/PyAgweB0aMZtoioarGHi4hMklYLjBkDbNwovR4wAFi5EtBolK2LiIwTAxcRmZzjx6WAdf48YGkJfPYZ8MYb7NUiourDwEVEJkMI4LvvgAkTgLw8wNsb2LABCAxUujIiMnYcw0VEJiE3Fxg5UrqMmJcnzR5/9CjDFhHVDAYuIjJ6Z85IwWr1asDMDPj0U+C33wAnJ6UrIyJTwUuKRGTU1q0Dxo4FsrMBd3fp9dNPK10VEZka9nARkVHKz5fWPxw0SApbzzwDHDvGsEVEymDgIiKjk5QEdO4MLFsmvX7vPSAyUurhIiJSAi8pEpFR2boVGD4cyMiQxmitWQP07Kl0VURk6tjDRURGobAQmDYN6NNHCluPPy5dQmTYIiJDwB4uIqr1/vkHePllYP9+6fXkycDcuYCVlbJ1ERGVYOAiolotMhIYMgS4dk1almfVKqBfP6WrIiLSx0uKRFQrFRcDH3wAhIRIYattW2kiU4YtIjJE7OEiolonPR145RWpdwsAxo0DFi4ErK2VrYuI6F4YuIioVtm/Hxg4ELh6FbC1Bb7+WgpfRESGjJcUiahWEAKYPx/o2lUKW02bAn/+ybBFRLUDe7iIyODdugWMGCHNsQVIg+SXLwfs7BQti4iowtjDRUQGSwhgwwagdWspbKnV0iXEH35g2CKi2oU9XERkkI4eBSZNAv73P+m1ry+wcSPQrp2iZRERVQp7uIjIoKSmAqNHAx06SGHLxgaYPRtISGDYIqLaiz1cRGQQ8vOBBQuATz4BsrKkbYMHSzPGe3kpWhoR0SNj4CIiRQkBbNkCvP02cPGitK1jR2leraAgRUsjIqoyvKRIRIr56y+ge3dpdviLFwEPD2D1auDgQYYtIjIuDFxEVOOuXQNefVVajmfvXunuw3ffBc6eBYYNA8z4m4mIjAwvKRJRjSkoABYtAj78ENBqpW0vvgjMmwf4+ChaGhFRtWLgIqJqJwSwbRvw1lvAuXPStrZtpUHyTz2laGlERDWCHfdEVK1OngRCQoD//EcKW66uwLffSsvyMGwRkalg4CKianHjBjBhgjRLfGQkYGUFTJsmha7RowFzc6UrJCKqObykSERVqrAQWLZMmqz01i1p2wsvSAtPN26saGlERIph4CKiKhMRAUyZApw+Lb1u1Qr48kugWzdl6yIiUhovKRLRI0tMBEJDgV69pLDl7AwsXy6th8iwRURk4IFr9uzZUKlUeo+AgAB5f15eHsLCwuDk5AQ7Ozv0798faWlpesdITk5GaGgobG1t4erqiqlTp6KoqEivTXR0NNq1awe1Wg0/Pz+Eh4fXxOkR1Xq3bgGTJwMtWgA7dgAWFlIP17lz0jxbHKdFRCQx6MAFAM2bN0dKSor82L9/v7xv8uTJ+O2337Bx40bExMTg6tWr6Nevn7y/uLgYoaGhKCgowIEDB7B69WqEh4dj5syZcpukpCSEhobimWeeQXx8PCZNmoQxY8Zg165dNXqeRLVJUZE0Tuuxx6SpHYqKgOefl+5I/PxzwNFR6QqJiAyMMGCzZs0SrVu3LndfRkaGsLS0FBs3bpS3nT59WgAQsbGxQgghduzYIczMzERqaqrcZtmyZUKj0Yj8/HwhhBDTpk0TzZs31zv2wIEDRUhIyEPVmpmZKQCIzMzMh3ofUW0TGSlEixZCSLNrCdGsmRC7dildFRFR5dTU97fB93CdO3cOnp6e8PX1xZAhQ5CcnAwAiIuLQ2FhIYKDg+W2AQEB8Pb2RmxsLAAgNjYWLVu2hJubm9wmJCQEWq0WJ0+elNuUPkZJm5Jj3Et+fj60Wq3eg8iYnT8P9OkDPPsscOIEUK+eNGv88eNAjx5KV0dEZNgMOnAFBgYiPDwcERERWLZsGZKSktClSxdkZWUhNTUVVlZWcLzr2oWbmxtSU1MBAKmpqXphq2R/yb77tdFqtbh9+/Y9a5szZw4cHBzkh5eX16OeLpFByswEpk4FmjUDtm6VxmW98YY0TmvCBGncFhER3Z9B/6rs1auX/LxVq1YIDAxEw4YNsWHDBtjY2ChYGTBjxgxMmTJFfq3Vahm6yKgUFwPffQe89x6Qni5tCwkBvvhCCl9ERFRxBt3DdTdHR0c0adIE58+fh7u7OwoKCpCRkaHXJi0tDe7u7gAAd3f3Mnctlrx+UBuNRnPfUKdWq6HRaPQeRMYiJgbo0AEYN04KW/7+wPbtwM6dDFtERJVRqwJXdnY2Lly4AA8PD7Rv3x6WlpaIioqS9ycmJiI5ORlBQUEAgKCgICQkJCC95H/PAURGRkKj0aDZv98aQUFBescoaVNyDCJTkpQEDBgAdO0KxMdLdxt++SWQkAA89xygUilcIBGZnLTsNPyW+Bve3/M+fjn1i9LlVJpBX1J8++230bt3bzRs2BBXr17FrFmzYG5ujkGDBsHBwQGjR4/GlClTUK9ePWg0GrzxxhsICgrC448/DgDo0aMHmjVrhqFDh2LevHlITU3Fe++9h7CwMKjVagDA+PHjsXjxYkybNg2jRo3Cnj17sGHDBmzfvl3JUyeqUVlZwJw50uXC/HzAzEyaR+vDD6VJTImIakJOQQ7iUuJw+J/D8uNy5mV5/4vNXkT/Zv0VrLDyDDpw/f333xg0aBBu3LgBFxcXPPnkkzh48CBcXFwAAF9++SXMzMzQv39/5OfnIyQkBEuXLpXfb25ujm3btuG1115DUFAQ6tSpg+HDh+PDDz+U2zRq1Ajbt2/H5MmTsXDhQjRo0ADffvstQkJCavx8iWqaTgd8/z0wYwbw730k6N5d6tVq2VLZ2ojIuBXpinDq2ikc+vuQFK6uHsaJ9BPQCZ1eOxVUaOrSFJ3qd0IP39p7S7RKCCGULsIYaLVaODg4IDMzk+O5qFb44w9g0iTgyBHpdePG0qSl//kPLx0SUdUSQiA5M1nutTr0zyHEpcQhtzC3TNv69vXRqX4ndKrfCYH1A9Hesz006ur7Xq2p72+D7uEioqqXnAxMnw6sWye91miA99+Xpnr490o7EdEjuXX7Fv68+qfepcG0nLQy7eyt7NGxfkd08uwkh6z6mvoKVFz9GLiITERODvB//wfMnw/k5Um9WGPGAB99BNw1FR0RUYXlF+XjeNpx6dLgVSlcnb1xtkw7CzMLtHJrhcD6gXK48nfyh7mZaSy6ysBFZOR0OmDtWuCdd4B//pG2Pf20tAZimzZKVkZEtY1O6HDuxjn5suDhfw4jPjUehbrCMm0b120sXxbsVL8T2ri3gY2lsnNoKomBi8iIHToETJwo/QkAPj7AZ58B/fpxnBYRPVhqdqreZcE/r/6JjLyMMu2cbZ2lXqtSlwadbJ1qvmADxsBFZIT++Ufq0VqzRnpdpw7w7rvA5MmAtbWytRGRYcouyEbc1Tj5jsHD/xxGcmZymXbWFtZo79FeDlad6ndCI8dGUPH/4u6LgYvIiNy+LfVgzZ0L5P5788+IEcCnnwIeHoqWRkQGpEhXhJPpJ/UuDZ68drLcKRmauTTTuzTYwrUFLM0tFaq89mLgIjICQgAbNgDTpkl3IQJA587SOK0OHRQtjYgUJoTA5czL+lMyXI3D7aLbZdo20DSQLw0GNghEe4/2sFfbK1C18WHgIqrl4uKk+bT275dee3kB8+YBAwdynBaRKbp1+9adcVf/XhpMz0kv006j1qCjZ0e9S4Oe9p4KVGwaGLiIaqnUVOC//wXCw6UeLltbaX6tt9+WnhOR8csrysPx1OPyZcHD/xzGuZvnyrSzMLNAG/c2eoPa/Z39YaaqVUsq12oMXES1TF6edKnwk0+A7Gxp25Ah0ritBg0ULY2IqpFO6HD2xlm9S4PHU4+XOyWDXz2/MlMyWFvwjhklMXAR1RJCAJs3Sz1YSUnStk6dgIULgX/XayciI1BYXIikjCQkXk/EmetnkHgjEYk3EpGQloDM/Mwy7Z1tneVgFVg/EB08O3BKBgPEwEVUC8THS1M6REdLrz09pVnjBw8GzHhFgKhWupF7A4k3/g1V1xPlYHX+5nkU6YrKfY+NhQ3ae7bXuzTo4+jDKRlqAQYuIgNVVATs2AF88430pxDSHFpTp0pjterUUbpCInqQwuJCXLx18U5P1b/B6sz1M7hx+8Y932djYQN/Z3/4O0mPAOcANHVpiuYuzTklQy3FwEVkYC5dAr79Fli1Crh69c72gQOlXq2GDRUrjYju4Xru9TI9VWeun8HFWxfv2VsFAF4aLzlYBTgHSAHL2R8NNA04oN3IMHARGYCCAmDrVmDFCiAyUurNAgBnZ2ni0jFjAH9/RUskMnkFxQW4cPNCmZ6qxBuJuHn75j3fZ2tpKwepAKcAOWA1cWqCOlbsqjYVDFxECjp3TurNCg8H0ktNkxMcDIwdC/TpA6jVipVHZHKEELiWe+1OT9X1RJy5IfVcXbx1EcWi+J7v9XbwLtNTFeAcgPr29TnGihi4iGpafj6waZPUm7V3753t7u7AyJHA6NFA48bK1UdkCgqKC3D+5vkyPVWJ1xNxK+/WPd9Xx7KOHKRKj696zOkx2FpyAjy6NwYuohpy+rQUsr7/Hrjx71hZlQro2VPqzXr+ecCSY2GJqowQAuk56Xd6qkpNsXDx1sUy6waWUEEFbwfvMj1V/k7+8LT3ZG8VVQoDF1E1un0b2LhRClolS+8A0gSlo0ZJDw6CJ3o0+UX5Um/VXT1VZ66fKXfeqhJ2VnZleqr8nf3xWL3HYGNpU4NnQKaAgYuoGvz1lxSy1qwBMjKkbebmQGgoMG6c1Ktlbq5oiUS1ihACaTlpZXqqzlw/g0sZl+7bW+Xj6FPunYAedh7sraIaw8BFVEWys4H166WgdejQne0NG0qXDEeOlCYsJaLyFRYXIjkzGRduXcDFWxdx8dZFvefafO0936tRa8q9E9Cvnh97q8ggMHARPaK4OClkrV0LZGVJ2ywspDsMx42T7jjkbPBEklu3b+mFqNKhKjkz+Z49VQBgpjKTeqvu6qnyd/KHu507e6vIoDFwEVWCVisFrBUrgKNH72z385N6s4YPB9zclKuPSClFuiJcybxSpneq5HlGXsZ9329tYQ3fur5oXLcxfOv6ys8b1W0E37q+XICZai0GLqIKEkK6VLhiBbBuHZCbK223sgL695eC1tNPszeLjF9mXma5l/wu3LqAyxmX7ztXFQC427nrBarSAYs9VWSsGLiIHuDWLWnw+4oVQELCne0BAdIlw6FDpRnhiYxFsa4Yf2v/vmeout+s6gCgNlejUd1G5YYqH0cfzq5OJomBi6gcQkjTOKxYIU3rkJcnbbe2Bl56SerN6txZmkeLqDbKys8qdxzVxVsXcSnjEgp1hfd9v2sd13Iv/fnW9YWHvQfXASS6CwMXUSnXr0sTk65YAZw5c2d7y5ZSb9aQIUDdusrVR1RROqHD1ayruHCz/Dv+ruVeu+/7Lc0s5XFT5Y2nsrOyq6EzITIODFxk8nQ6IDpaClmbNkkLSQNAnTrAyy9LvVmdOrE3iwxPTkEOkjKS9ELVxYyLuHDzApIyklBQXHDf9zvbOpcJVCWvPe09YW7GyeKIqgoDF5mstDRp0ehvvwXOn7+zvX17KWQNGgRoNIqVRyZOCIGMvAykZKcgJSvlzpiqjH97q25eQFpO2n2PYWFmAR9Hn3JDlW9dX2jU/AdOVFMYuMik6HRAZKTUm/Xrr0BRkbTd3l66XDh2LNCunbI1knHTCR1u5N5ASnYKrmZdRUpWihyqUrJT5O2p2anIK8p74PHq2dQrM4aq5NFA0wAWZvw1T2QI+F8imYR//gFWrZJ6sy5fvrP98celkDVwoHQJkaiyinXFSM9Jl0JU6QCVdSdEpWSnIDU7FUW6ogoft651XXjYe8DT3hO+jr5oXE8/VDlaO1bfSRFRlWHgIqNVVARERADffANs3y71bgGAo6M0lcPYsdJgeKL7KSguQGp26j0DVMnr9Jz0+86SfjcXWxd42nvCw94DHnb/Pv4NViXP3e3cOdEnkZFg4CKjc/ky8N13wMqVUs9WiS5dpJA1YABgw6XVTN7twttlA1Spy3olz6/nXq/wMc1UZnC3c5cD072ClFsdN1iaW1bj2RGRoWHgIqNQWAhs2yb1Zu3aJc2jBQBOTtIyO2PGAE2bKlsj1Yys/Cw5MJW5vFcqWGXmZ1b4mJZmlncCVKkgpddDZe8BF1sX3tlHROVi4KJa7eJFaVzWqlVAauqd7d26SfNm9e0LqNWKlUdVpOSOvQeNj0rJSkFOYU6Fj2tjYVMmSJXuiSr508nGicvNENEjYeCiWic/X7rD8JtvgKioO9vd3IARI6TeLD8/xcqjh5BdkI3U7FSkZachNTtVep5T9nladhryi/MrfFx7K/sKBSkHtQODFBHVCAYuqjUSE6XerPBwaUZ4QJqMtEcPqTerd2/AksNiFJdXlPfAAFUSoh6mNwq4c8eePC7KzrPspT57D86CTkQGh4GLDI4QwM2bwKVL0iMpCfjtN2DfvjttPD2B0aOBUaMAHx+FCjUhhcWFSM9Jf2AvVGp26kONjQIAW0tbuNu5yw+3Om5lnrvZufGOPSKq1Ri4qMbdHajKe2Rnl32fmRnw3HNSb1avXoAF//U+kmJdMa7nXi+35yk1J1UvRN24feOhjq02V8shqbwAVTpEsTeKiEwBv7KoylU2UN3Nw0PqvfLxkebLGjoUaNCg2so2CkII3Lx9854hqvSlvmu51x5q3ihzlXmFQxTHRhER6WPgoodWHYHq7oe3N2DNq0cApJ6orIIs/Ut69xgjlZ6TjkJdYYWPrYIKLnVc9C/flROg3O3cUc+mHsxUZtV4pkRExouBi8pgoKq8Yl0xsguyyzyyCrLK356fhezCstvlfQXZuF10+6HrqGdT74G9UO527nC2deZae0RENYC/aU0QA5WkSFeEnIKce4ah0qFHfhTeY/u/j8qEo4rSqDUVClGudVxhZW5VbXUQEdHDY+AyQsYYqIp0RQ8OQw/qTbpre15RXrXVa64yh73aHnZWdrCzsoO91Z3npR/33K4uu5136BER1V4MXHdZsmQJ5s+fj9TUVLRu3RqLFi1Cp06dlC5LT00FKrVaoEhXhPzifOQV5SG/6N8/i/Nx6pb0urx9d78us68ibe56XaQrqpa/SwCwMLO4f/CxrFggKh2grMytOGiciIhkDFylrF+/HlOmTMHy5csRGBiIBQsWICQkBImJiXB1dVWkpqMJudgUcR2X/s7DlZR8/J2ah5T0fNwuyAcs8gCLf/80//dPu3ygjfS6jmM+NHXzYOeQDxtNHqzr5MPKNg+W1vlQWeahUEiB6WRRHo4W5SMvNw/5CfnIO3Yn8AgIRc77Xu4OR2WCj2XFAlHpB8MRERFVN5UQwrC+URUUGBiIjh07YvHixQAAnU4HLy8vvPHGG3jnnXfu+16tVgsHBwdkZmZCo9FUWU3jl67B19eGVtnxHoWFmQWsLayhNldLf1qo9Z7fc19F2jxgn42FDezV9hybREREVaq6vr/vxh6ufxUUFCAuLg4zZsyQt5mZmSE4OBixsbFl2ufn5yM//87ablqttlrq8vOxgVmaGpYqa1iZq2FjYQ1btRp11GrYWN0nzJiryw0xlQ08anM1zM3Mq+UciYiIjB0D17+uX7+O4uJiuLm56W13c3PDmTNnyrSfM2cOPvjgg2qv6+3n+uPt5/pX++cQERFR9eEshpU0Y8YMZGZmyo8rV64oXRIREREZKPZw/cvZ2Rnm5uZIS0vT256WlgZ3d/cy7dVqNdRqdU2VR0RERLUYe7j+ZWVlhfbt2yMqKkreptPpEBUVhaCgIAUrIyIiotqOPVylTJkyBcOHD0eHDh3QqVMnLFiwADk5ORg5cqTSpREREVEtxsBVysCBA3Ht2jXMnDkTqampaNOmDSIiIsoMpCciIiJ6GJyHq4rU1DweREREVHVq6vubY7iIiIiIqhkDFxEREVE1Y+AiIiIiqmYMXERERETVjIGLiIiIqJoxcBERERFVMwYuIiIiomrGwEVERERUzTjTfBUpmT9Wq9UqXAkRERFVVMn3dnXPA8/AVUWysrIAAF5eXgpXQkRERA8rKysLDg4O1XZ8Lu1TRXQ6Ha5evQp7e3uoVCqly6kSWq0WXl5euHLlisktV8Rz57nz3E0Hz920zz05ORkqlQqenp4wM6u+kVbs4aoiZmZmaNCggdJlVAuNRmNy/yGW4Lnz3E0Nz53nbmocHBxq5Nw5aJ6IiIiomjFwEREREVUzBi66J7VajVmzZkGtVitdSo3jufPcTQ3Pneduamr63DlonoiIiKiasYeLiIiIqJoxcBERERFVMwYuIiIiomrGwEVERERUzRi4jNy+ffvQu3dveHp6QqVSYcuWLXr7hRCYOXMmPDw8YGNjg+DgYJw7d06vzc2bNzFkyBBoNBo4Ojpi9OjRyM7O1mvz119/oUuXLrC2toaXlxfmzZtX3af2QHPmzEHHjh1hb28PV1dX9O3bF4mJiXpt8vLyEBYWBicnJ9jZ2aF///5IS0vTa5OcnIzQ0FDY2trC1dUVU6dORVFRkV6b6OhotGvXDmq1Gn5+fggPD6/u07uvZcuWoVWrVvJkhkFBQdi5c6e831jP+25z586FSqXCpEmT5G3GfO6zZ8+GSqXSewQEBMj7jfnc//nnH7zyyitwcnKCjY0NWrZsiSNHjsj7jfV3nY+PT5mfuUqlQlhYGADj/pkXFxfj/fffR6NGjWBjY4PGjRvjo48+0lsT0aB+7oKM2o4dO8S7774rNm3aJACIzZs36+2fO3eucHBwEFu2bBHHjx8X//nPf0SjRo3E7du35TY9e/YUrVu3FgcPHhT/+9//hJ+fnxg0aJC8PzMzU7i5uYkhQ4aIEydOiJ9++knY2NiIr7/+uqZOs1whISFi1apV4sSJEyI+Pl4899xzwtvbW2RnZ8ttxo8fL7y8vERUVJQ4cuSIePzxx8UTTzwh7y8qKhItWrQQwcHB4tixY2LHjh3C2dlZzJgxQ25z8eJFYWtrK6ZMmSJOnTolFi1aJMzNzUVERESNnm9pW7duFdu3bxdnz54ViYmJ4r///a+wtLQUJ06cEEIY73mXdvjwYeHj4yNatWolJk6cKG835nOfNWuWaN68uUhJSZEf165dk/cb67nfvHlTNGzYUIwYMUIcOnRIXLx4UezatUucP39ebmOsv+vS09P1ft6RkZECgNi7d68Qwnh/5kII8cknnwgnJyexbds2kZSUJDZu3Cjs7OzEwoUL5TaG9HNn4DIhdwcunU4n3N3dxfz58+VtGRkZQq1Wi59++kkIIcSpU6cEAPHnn3/KbXbu3ClUKpX4559/hBBCLF26VNStW1fk5+fLbaZPny78/f2r+YweTnp6ugAgYmJihBDSuVpaWoqNGzfKbU6fPi0AiNjYWCGEFFjNzMxEamqq3GbZsmVCo9HI5ztt2jTRvHlzvc8aOHCgCAkJqe5Teih169YV3377rUmcd1ZWlnjsscdEZGSkePrpp+XAZeznPmvWLNG6dety9xnzuU+fPl08+eST99xvSr/rJk6cKBo3bix0Op1R/8yFECI0NFSMGjVKb1u/fv3EkCFDhBCG93PnJUUTlpSUhNTUVAQHB8vbHBwcEBgYiNjYWABAbGwsHB0d0aFDB7lNcHAwzMzMcOjQIbnNU089BSsrK7lNSEgIEhMTcevWrRo6mwfLzMwEANSrVw8AEBcXh8LCQr3zDwgIgLe3t975t2zZEm5ubnKbkJAQaLVanDx5Um5T+hglbUqOobTi4mKsW7cOOTk5CAoKMonzDgsLQ2hoaJn6TOHcz507B09PT/j6+mLIkCFITk4GYNznvnXrVnTo0AEvvvgiXF1d0bZtW6xYsULebyq/6woKCrBmzRqMGjUKKpXKqH/mAPDEE08gKioKZ8+eBQAcP34c+/fvR69evQAY3s+dgcuEpaamAoDef2glr0v2paamwtXVVW+/hYUF6tWrp9emvGOU/gyl6XQ6TJo0CZ07d0aLFi0ASLVZWVnB0dFRr+3d5/+gc7tXG61Wi9u3b1fH6VRIQkIC7OzsoFarMX78eGzevBnNmjUz+vNet24djh49ijlz5pTZZ+znHhgYiPDwcERERGDZsmVISkpCly5dkJWVZdTnfvHiRSxbtgyPPfYYdu3ahddeew1vvvkmVq9eDcB0ftdt2bIFGRkZGDFiBADj//f+zjvv4OWXX0ZAQAAsLS3Rtm1bTJo0CUOGDAFgeD93i4c4N6JaKywsDCdOnMD+/fuVLqXG+Pv7Iz4+HpmZmfj5558xfPhwxMTEKF1Wtbpy5QomTpyIyMhIWFtbK11OjSv5P3sAaNWqFQIDA9GwYUNs2LABNjY2ClZWvXQ6HTp06IBPP/0UANC2bVucOHECy5cvx/DhwxWuruasXLkSvXr1gqenp9Kl1IgNGzbgxx9/xNq1a9G8eXPEx8dj0qRJ8PT0NMifO3u4TJi7uzsAlLljJS0tTd7n7u6O9PR0vf1FRUW4efOmXpvyjlH6M5Q0YcIEbNu2DXv37kWDBg3k7e7u7igoKEBGRoZe+7vP/0Hndq82Go1G0S85Kysr+Pn5oX379pgzZw5at26NhQsXGvV5x8XFIT09He3atYOFhQUsLCwQExODr776ChYWFnBzczPacy+Po6MjmjRpgvPnzxv1z93DwwPNmjXT29a0aVP5cqop/K67fPkyfv/9d4wZM0beZsw/cwCYOnWq3MvVsmVLDB06FJMnT5Z7tw3t587AZcIaNWoEd3d3REVFydu0Wi0OHTqEoKAgAEBQUBAyMjIQFxcnt9mzZw90Oh0CAwPlNvv27UNhYaHcJjIyEv7+/qhbt24NnU1ZQghMmDABmzdvxp49e9CoUSO9/e3bt4elpaXe+ScmJiI5OVnv/BMSEvT+g4yMjIRGo5F/wQcFBekdo6RNyTEMhU6nQ35+vlGfd/fu3ZGQkID4+Hj50aFDBwwZMkR+bqznXp7s7GxcuHABHh4eRv1z79y5c5kpX86ePYuGDRsCMP7fdQCwatUquLq6IjQ0VN5mzD9zAMjNzYWZmX6MMTc3h06nA2CAP/eHGmJPtU5WVpY4duyYOHbsmAAgvvjiC3Hs2DFx+fJlIYR0y6yjo6P49ddfxV9//SX69OlT7i2zbdu2FYcOHRL79+8Xjz32mN4tsxkZGcLNzU0MHTpUnDhxQqxbt07Y2toqPi3Ea6+9JhwcHER0dLTebdO5ublym/Hjxwtvb2+xZ88eceTIEREUFCSCgoLk/SW3TPfo0UPEx8eLiIgI4eLiUu4t01OnThWnT58WS5YsUfyW6XfeeUfExMSIpKQk8ddff4l33nlHqFQqsXv3biGE8Z53eUrfpSiEcZ/7W2+9JaKjo0VSUpL4448/RHBwsHB2dhbp6elCCOM998OHDwsLCwvxySefiHPnzokff/xR2NraijVr1shtjPl3XXFxsfD29hbTp08vs89Yf+ZCCDF8+HBRv359eVqITZs2CWdnZzFt2jS5jSH93Bm4jNzevXsFgDKP4cOHCyGk22bff/994ebmJtRqtejevbtITEzUO8aNGzfEoEGDhJ2dndBoNGLkyJEiKytLr83x48fFk08+KdRqtahfv76YO3duTZ3iPZV33gDEqlWr5Da3b98Wr7/+uqhbt66wtbUVL7zwgkhJSdE7zqVLl0SvXr2EjY2NcHZ2Fm+99ZYoLCzUa7N3717Rpk0bYWVlJXx9ffU+QwmjRo0SDRs2FFZWVsLFxUV0795dDltCGO95l+fuwGXM5z5w4EDh4eEhrKysRP369cXAgQP15qIy5nP/7bffRIsWLYRarRYBAQHim2++0dtvzL/rdu3aJQCUOR8hjPtnrtVqxcSJE4W3t7ewtrYWvr6+4t1339WbvsGQfu4qIUpNyUpEREREVY5juIiIiIiqGQMXERERUTVj4CIiIiKqZgxcRERERNWMgYuIiIiomjFwEREREVUzBi4iIiKiasbARURERFTNGLiIiErp2rUrJk2aBADw8fHBggULFK2HiIyDhdIFEBEZqj///BN16tRRugwiMgIMXERE9+Di4qJ0CURkJHhJkYhMVk5ODoYNGwY7Ozt4eHjg888/19t/9yVFlUqFr7/+Gs8//zxsbW3RtGlTxMbG4vz58+jatSvq1KmDJ554AhcuXKjhMyEiQ8fARUQma+rUqYiJicGvv/6K3bt3Izo6GkePHr3vez766CMMGzYM8fHxCAgIwODBg/Hqq69ixowZOHLkCIQQmDBhQg2dARHVFrykSEQmKTs7GytXrsSaNWvQvXt3AMDq1avRoEGD+75v5MiReOmllwAA06dPR1BQEN5//32EhIQAACZOnIiRI0dWb/FEVOuwh4uITNKFCxdQUFCAwMBAeVu9evXg7+9/3/e1atVKfu7m5gYAaNmypd62vLw8aLXaKq6YiGozBi4ioodgaWkpP1epVPfcptPparYwIjJoDFxEZJIaN24MS0tLHDp0SN5269YtnD17VsGqiMhYcQwXEZkkOzs7jB49GlOnToWTkxNcXV3x7rvvwsyM/x9KRFWPgYuITNb8+fORnZ2N3r17w97eHm+99RYyMzOVLouIjJBKCCGULoKIiIjImLHvnIiIiKiaMXARERERVTMGLiIiIqJqxsBFREREVM0YuIiIiIiqGQMXERERUTVj4CIiIiKqZgxcRERERNWMgYuIiIiomjFwEREREVUzBi4iIiKiavb/0yqqQmBByK8AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mul silu:\n",
      "      dim  Triton-FusedMulSiLU  HF-LlamaMulSiLU\n",
      "0   512.0            99.306673        67.722902\n",
      "1  1536.0           831.676662       238.228232\n",
      "2  2560.0          3152.609348       516.662121\n",
      "3  3584.0          6516.749382      1057.967663\n",
      "4  4608.0         10974.406242      1634.543896\n",
      "5  5632.0         16589.042664      2419.462204\n",
      "6  6656.0         23033.687592      3318.871021\n",
      "7  7680.0         30637.748718      4561.211586\n"
     ]
    }
   ],
   "source": [
    "\n",
    "torch.cuda.empty_cache()\n",
    "@triton.testing.perf_report(\n",
    "    triton.testing.Benchmark(\n",
    "        x_names=['dim'],  # argument names to use as an x-axis for the plot\n",
    "        x_vals=[512 * i for i in range(1, 16+1, 2)],  # different possible values for `x_name`\n",
    "        line_arg='provider',  # argument name whose value corresponds to a different line in the plot\n",
    "        line_vals=['Triton-FusedMulSiLU', 'HF-LlamaMulSiLU'],  # possible values for `line_arg``\n",
    "        line_names=[\n",
    "            \"Triton-FusedMulSiLU\",\n",
    "            \"HF-LlamaMulSiLU\",\n",
    "        ],  # label name for the lines\n",
    "        styles=[('blue', '-'), ('green', '-')],  # line styles\n",
    "        ylabel=\"ms\",  # label name for the y-axis\n",
    "        plot_name=\"mul silu\",  # name for the plot. Used also as a file name for saving the plot.\n",
    "        args={'seq_len': 128, 'bs': 8}\n",
    "        # args={'bs': 2, 'num_head': 32, 'rope_head_dim': 32, \n",
    "        #       'nope_head_dim': 64, 'kv_lora_rank': 256},  # values for function arguments not in `x_names` and `y_name`\n",
    "    ))\n",
    "def benchmark(bs, seq_len, dim, provider):\n",
    "    device = torch.device('cuda')\n",
    "    dtype = torch.float16\n",
    "    tensor = torch.randn(bs, seq_len, dim).to(device).to(dtype)\n",
    "    stream = torch.cuda.Stream()\n",
    "    torch.cuda.set_stream(stream)\n",
    "    if provider == 'Triton-FusedMulSiLU':\n",
    "        func = TritonFusedMulSiLU(dim).cuda().to(dtype)\n",
    "        ms = triton.testing.do_bench(lambda: func(tensor))\n",
    "    if provider == 'HF-LlamaMulSiLU':\n",
    "        func = MulSiLU(dim).cuda().to(dtype)\n",
    "        ms = triton.testing.do_bench(lambda: func(tensor))\n",
    "\n",
    "    return ms * 1e3\n",
    "print(f'bs: {8}, seq_len: {1024}')\n",
    "benchmark.run(show_plots=True, print_data=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
